# -*- coding: UTF-8 -*-

import sys
import os
import urllib2
import urlparse
import bs4
import argparse
import time

#from urllib2 import urlopen
from bs4 import BeautifulSoup
#from bs4 import SoupStrainer

URLscheme = u"http"
URLsite = u"www.xxxx.com"
URLpath = u"3y8kh/7"

out_dir = u'D:/Downloads/book/test/'
tmp_file = 'temp.txt'

def c10tc_linkpage_parser(rooturl):
    response = urllib2.urlopen(rooturl)
    page = response.read()
    response.close()
    
    soup = BeautifulSoup(page)
    
    allasoup = soup.find_all('a')
    
    linkList = []
    indexList = []
    
    for eachasoup in allasoup:
        urlstr = eachasoup.get('href')
        if (urlstr.count(URLpath) > 0):
            if (urlstr.count('index') > 0):
                indexList.append(urlstr)
            else:
                if (urlstr.count('.html') > 0):
                    linkList.append(urlstr)
    
    '''
    print 'All links in page ', rooturl
    for eachlink in linkList:
        print eachlink
    
    print 'All index in page ', rooturl
    for eachindex in indexList:
        print eachindex
    ''' 
    return (linkList, indexList)    


def c10tc_txtpage_downloader(url):
    response = urllib2.urlopen(url)
    page = response.read()    
    response.close()
    
    soup = BeautifulSoup(page, from_encoding="gb18030")
    
    # get file title
    title = str(soup.title).decode('UTF-8').strip('<title>').strip('</title>').partition(u' ')[0]
    #print ('title = ', title)
       
    #get file content
    allpsoup = soup.find_all('p')
           
    outfile = open(tmp_file, 'w')
    
    for eachpsoup in allpsoup:        
        outfile.write(str(eachpsoup).replace('<br/>', ''))
        
    outfile.close()
    
    return title

def GenerateFinalFileWithName(tmpfilename, finalfilename):
    tmpfile = open(tmpfilename, 'r')
    alllines = tmpfile.readlines()    
    outfile = open(finalfilename, 'w')
    
    # remove the line if it contains the '<p>'
    for eachline in alllines:
        if str(eachline).count('<p>') == 0:            
            outfile.write(eachline)
            
    tmpfile.close()
    outfile.close() 
    
    # remove the temp file
    os.remove(tmpfilename)

def GenerateFinalFile(tmpfilename):
    tmpfile = open(tmpfilename, 'r')
    alllines = tmpfile.readlines()
    i = 0
    
    # remove the line if it contains the '<p>'
    for eachline in alllines:
        if str(eachline).count('<p>') == 0:            
            # get the file name
            if (i == 0) and (len(str(eachline).strip()) > 0):
                filename = eachline.decode('UTF-8').strip() + '.txt'
                outfile = open(filename, 'w')
                i = 1
            if (i == 1):
                outfile.write(eachline)
            
    tmpfile.close()
    outfile.close() 
    
    # remove the temp file
    os.remove(tmpfilename)
    
    #print ('filename = ', filename)
        

if __name__ == '__main__':    
    #current_encoding = sys.getfilesystemencoding()
    #print (current_encoding)
    
    argument_parser = argparse.ArgumentParser(description='text file downloader')
    argument_parser.add_argument('-url', dest = 'StartURL', help='the start url to download')
    argument_parser.add_argument('-path', dest = 'URLpath', help='the start path to download')
    
    args = argument_parser.parse_args()    
    #print args.StartURL
    
    URLpath = args.URLpath
    
    URLopen = args.StartURL    
    linkList, indexList = c10tc_linkpage_parser(URLopen)
        
    for eachlink in linkList:
        URLopen = urlparse.urlunparse((URLscheme, URLsite, eachlink, '', '', ''))
        print URLopen
        
        time.sleep(1.0)
        
        try:
            title = c10tc_txtpage_downloader(URLopen)
            outfilename = out_dir + title + u'.txt'
            
            # generate the final text file
            GenerateFinalFileWithName(tmp_file, outfilename)        
        except:
            print "Unexpected error:", sys.exc_info()[0]
     

